
#!pip install pyspark


#!pip install wordcloud


import plotly.express._doc as xpdocs
import matplotlib.pyplot as plt


from itables import init_notebook_mode
import itables.options as opt
init_notebook_mode(all_interactive=True)


%config InlineBackend.figure_format = 'retina'


opt.column_filters= 'footer'
opt.classes = 'display nowrap cell-border'
opt.dom = 'lftipr'
opt.search = {"regex": True, "caseInsensitive": True, "smart":True, 'highlight':True}
opt.paging = True
opt.autoWidth=False
opt.showIndex = False
opt.columnDefs=[{"width": "120px", "targets": "_all"}]


from pyspark.sql.functions import *
from pyspark.sql.types import *
from pyspark.sql import SparkSession
#Create Spark Session
spark = SparkSession.builder.getOrCreate()

# File location and type
file_location = "netflix_titles.csv"

# Define the schema
schema = StructType([
    StructField("show_id", IntegerType()),
    StructField("type", StringType()),
    StructField("title", StringType()),
    StructField("director", StringType()),
    StructField("cast", StringType()),
    StructField("country", StringType()),
    StructField("date_added", DateType()),
    StructField("release_year", IntegerType()),
    StructField("rating", StringType()),
    StructField("duration", StringType()),
    StructField("listed_in", StringType())
])

# The applied options are for CSV files. For other file types, these will be ignored.
spark_data = spark.read.format('csv') \
                  .option("inferSchema", 'false') \
                  .option("header", 'true') \
                  .option("dateFormat", "MMMM d, yyyy")\
                  .schema(schema) \
                  .load(file_location)


opt.maxBytes = spark_data.toPandas().memory_usage().sum()


display(spark_data.toPandas())


import pandas as pd

url = 'netflix_titles.csv'
pandas_data = df = pd.read_csv(url, parse_dates=['date_added', 'release_year'])


#pandas_data['date_added'] = pd.to_datetime(pandas_data.date_added, format='%B %d, %Y').dt.date
pandas_data['release_year'] = pd.to_datetime(pandas_data.release_year).dt.year


opt.maxBytes = pandas_data.memory_usage().sum()
pandas_data


pandas_data.columns

Index(['show_id', 'type', 'title', 'director', 'cast', 'country', 'date_added',
       'release_year', 'rating', 'duration', 'listed_in', 'description'],
      dtype='object')


spark_data.printSchema()

root
 |-- show_id: integer (nullable = true)
 |-- type: string (nullable = true)
 |-- title: string (nullable = true)
 |-- director: string (nullable = true)
 |-- cast: string (nullable = true)
 |-- country: string (nullable = true)
 |-- date_added: date (nullable = true)
 |-- release_year: integer (nullable = true)
 |-- rating: string (nullable = true)
 |-- duration: string (nullable = true)
 |-- listed_in: string (nullable = true)


pandas_data.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 6234 entries, 0 to 6233
Data columns (total 12 columns):
 #   Column        Non-Null Count  Dtype 
---  ------        --------------  ----- 
 0   show_id       6234 non-null   int64 
 1   type          6234 non-null   object
 2   title         6234 non-null   object
 3   director      4265 non-null   object
 4   cast          5664 non-null   object
 5   country       5758 non-null   object
 6   date_added    6223 non-null   object
 7   release_year  6234 non-null   int32 
 8   rating        6224 non-null   object
 9   duration      6234 non-null   object
 10  listed_in     6234 non-null   object
 11  description   6234 non-null   object
dtypes: int32(1), int64(1), object(10)
memory usage: 560.2+ KB


pandas_data['show_id'] = pandas_data.show_id.astype('object')


spark_data_nulls = spark_data.select([count(when(col(c).isNull(), c)).alias(c) for c in spark_data.columns])
spark_nulls = spark_data_nulls.toPandas()
missing_spark_df = pd.DataFrame()
missing_spark_df['category'] = spark_nulls.columns
missing_spark_df['total'] = spark_nulls.iloc[0].values
missing_spark_df['percentage'] = missing_spark_df['total']/ spark_data.count()
missing_spark_df = missing_spark_df.sort_values('total', ascending=0).round(4)


opt.paging=False
display(missing_spark_df)


from plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot 
init_notebook_mode(connected=False)
import plotly.express as px

scale = 3/4

reds = ['salmon','darksalmon','red','crimson','darkred']
reds2 = ['red', 'crimson', 'darkred']
item1 = px.bar(missing_spark_df, y='category', x='total', orientation='h',color= 'category', color_discrete_sequence=['red'], template='plotly_dark', height=625)

def update_style(item, title,size, h, w):
    fig = item.update_layout(height=h, width= w, title={'text': title, 'x':0.5, 'xanchor':'center', 'yanchor': 'top', 'font':{'size':size, 'color':'red', 'family':'Balto'}}, xaxis_showgrid=False, yaxis_showgrid=False, #plot_bgcolor='rgba(0, 0, 0, 0)', paper_bgcolor='rgba(0, 0, 0, 0)', 
                             template= 'plotly_dark')
    return fig
item1= update_style(item1, 'Spark null values', 40*scale, 600*scale, 1200*scale)
item1.show()


item2 = px.pie(missing_spark_df, values='total', names='category', hole=0.4, template='plotly_dark', color_discrete_sequence=reds[::-1])
item2 = update_style(item2, 'Total Null values percentage',40*scale, 600*scale, 1200*scale)
item2.show()


import matplotlib.pyplot as plt
from matplotlib.colors import LinearSegmentedColormap
from wordcloud import WordCloud

cmap = LinearSegmentedColormap.from_list("", ['#f00','#7a1b0c'])
spark_words = [(w, f) for w,f in zip(missing_spark_df.category, missing_spark_df.total)]
wordcloud = WordCloud(width = 1440, height = 190,
                background_color ='black',
                mode = "RGBA",
                colormap = cmap,
                min_font_size = 9,
                relative_scaling = 0.01,
                #stopwords = STOPWORDS,
                )
wordcloud.generate_from_frequencies(dict(spark_words))
# show words cloud
plt.figure(figsize = (10, 10), dpi= 300, facecolor = None)
plt.imshow(wordcloud)
plt.margins(x=0, y=0)
plt.axis("off")
plt.show()


bool_na = pandas_data.isna().sum() > 0
total = pandas_data.isna().sum()[bool_na]
missing_pandas_df= pd.DataFrame()
missing_pandas_df['category'] = total.index
missing_pandas_df['total'] = total.values
missing_pandas_df['percentage'] = total.values/ pandas_data.shape[0]
missing_pandas_df = missing_pandas_df.round(4)


display(missing_pandas_df)


item3 = px.bar(missing_pandas_df, y='category', x='total', orientation='h', color= 'category', color_discrete_sequence=['red'], template='plotly_dark')
item3 = update_style(item3, 'Pandas null values', 40*scale, 600*scale, 1200*scale)
item3.show()


item4 = px.pie(missing_pandas_df, values='total', names='category', hole=.4, opacity=.7, color_discrete_sequence= reds[::-1], template='plotly_dark')
item4 = update_style(item4, 'Total Null values percentage', 40*scale, 600*scale, 1200*scale)
item4.show()


cmap = LinearSegmentedColormap.from_list("", ['#f00','#7a1b0c'])
pandas_words = [(w, f) for w,f in zip(missing_pandas_df.category, missing_pandas_df.total)]
wordcloud = WordCloud(width = 1440, height = 190,
                background_color ='black',
                mode = "RGBA",
                colormap = cmap,
                min_font_size = 1,
                relative_scaling = 0.03,
                #stopwords = STOPWORDS,
                )
wordcloud.generate_from_frequencies(dict(pandas_words))
# show words cloud
plt.figure(figsize = (10, 10), dpi= 300, facecolor = None)
plt.imshow(wordcloud)
plt.margins(x=0, y=0)
plt.axis("off")
plt.show()


missing_spark_pandas = missing_spark_df.merge(missing_pandas_df, on='category', how='outer', suffixes=('_spark', '_pandas')).fillna(0)
missing_spark_pandas['difference'] = missing_spark_pandas.total_spark - missing_spark_pandas.total_pandas
missing_spark_pandas['total_cum_spark'] = missing_spark_pandas.total_spark.cumsum()
missing_spark_pandas['total_cum_pandas'] = missing_spark_pandas.total_pandas.cumsum()
missing_spark_pandas['diff_cum'] = missing_spark_pandas.difference.cumsum()
display(missing_spark_pandas)


import plotly.io as pio
pio.templates

Templates configuration
-----------------------
    Default template: 'plotly'
    Available templates:
        ['ggplot2', 'seaborn', 'simple_white', 'plotly',
         'plotly_white', 'plotly_dark', 'presentation', 'xgridoff',
         'ygridoff', 'gridon', 'none']


item5 = px.bar(missing_spark_pandas, x='category', y=['total_spark', 'total_pandas', 'difference'], orientation='v', barmode='group', color_discrete_sequence=reds2, template='plotly_dark')
item5 = update_style(item5, 'Total Null Values Comparision', 40*scale, 600*scale, 1200*scale)
item5.show()


item6 = px.line(missing_spark_pandas, x='category', y=['total_spark', 'total_pandas','difference'], color_discrete_sequence=reds2, template='presentation')
item6 = update_style(item6, 'Total Null Values Comparision', 40*scale, 600*scale, 1200*scale)
item6.show()


item7 = px.line(missing_spark_pandas, x='category', y=['total_cum_spark', 'total_cum_pandas'], color_discrete_sequence=reds2, template='presentation')
item7 = update_style(item7, 'Total Cumulative Null Values Comparision', 40*scale, 600*scale, 1400*scale)
item7.update_layout(template='presentation')
item7.show()


word= list(missing_spark_df.category.values)+list(missing_pandas_df.category.values)
freq= list(missing_spark_df.total.values)+list(missing_pandas_df.total.values)
cmap = LinearSegmentedColormap.from_list("", ['#f00','#7a1b0c'])
words = [(w, f) for w,f in zip(word, freq)]
wordcloud = WordCloud(width = 1440, height = 249,
                background_color ='black',
                mode = "RGBA",
                colormap = cmap,
                min_font_size = 1,
                relative_scaling = 0.03,
                #stopwords = STOPWORDS,
                )
wordcloud.generate_from_frequencies(dict(words))
# show words cloud
plt.figure(figsize = (10, 10), dpi= 300, facecolor = None)
plt.imshow(wordcloud)
plt.margins(x=0, y=0)
plt.axis("off")
plt.show()


data = pandas_data.copy()


item8 = px.imshow(data.isna(), color_continuous_scale=['red', 'black'],
                  template='plotly_dark', range_color=[0,1],
                    labels=dict(x="Columns", y="No. Record", color="Null Values"))

# Add color bar legend in range (0,1)
item8.update_layout(coloraxis_colorbar=dict(
    title="",
    tickvals=[0, 1],
    ticktext=["Not Nulls", "Nulls"],
    ticks="inside",
    thickness=15,
    tickfont=dict(size=15),
    outlinecolor="lightgray",
    outlinewidth=1
))
item8.update_layout(paper_bgcolor='rgba(0, 0, 0, 1)')

item8 = update_style(item8, 'Null Values Matrix', 40*scale,450, 1440*scale)
item8.show()


opt.paging = False
display(missing_pandas_df)


missing_types= data[missing_pandas_df.category].dtypes.reset_index()
missing_types.columns = ['category', 'data_type']
display(missing_types)


opt.paging= True
display(data[data.director.isna()])


data.director.fillna('no director', inplace=True)
display(data.director)


display(data[data.cast.isna()])


data.cast.fillna('no cast', inplace=True)
data.cast


display(data[data.country.isna()])


data.country.fillna('no country', inplace=True)


data.country


display(data[data.date_added.isna()])


data.dropna(subset=['date_added', 'rating'], inplace=True)
data[['date_added','rating']]


count_missing = data.isna().sum().reset_index()
count_missing.columns = ['category', 'null values sum']
display(count_missing)


item9 = px.imshow(data.isna(), color_continuous_scale=['red', 'black'],
                  template='plotly_dark', range_color=[0,1],
                    labels=dict(x="Columns", y="No. Record", color="Null Values"))

# Add color bar legend in range (0,1)
item9.update_layout(coloraxis_colorbar=dict(
    title="",
    tickvals=[0, 1],
    ticktext=["Not Nulls", "Nulls"],
    ticks="inside",
    thickness=15,
    tickfont=dict(size=15),
    outlinecolor="lightgray",
    outlinewidth=1
))
item9.update_layout(paper_bgcolor='rgba(0, 0, 0, 1)')

item9 = update_style(item9, 'Null Values Matrix', 40*scale,450, 1440*scale)
item9.show()


data.to_csv('netflix_titles_clean.csv', index=False)


!jupyter nbconvert --to html_toc --theme jupyterlab_miami_nights --output Netflix_Null_Values.html Netflix_Null_Values.ipynb

[NbConvertApp] Converting notebook Netflix_Null_Values.ipynb to html
[NbConvertApp] Writing 13047719 bytes to Netflix_Null_Values.html

show_id	type	title	director	cast	country	date_added	release_year	rating	duration	listed_in
Loading... (need help?)
show_id	type	title	director	cast	country	date_added	release_year	rating	duration	listed_in

category	total_spark	percentage_spark	total_pandas	percentage_pandas	difference	total_cum_spark	total_cum_pandas	diff_cum
Loading... (need help?)
category	total_spark	percentage_spark	total_pandas	percentage_pandas	difference	total_cum_spark	total_cum_pandas	diff_cum

Netflix Null Values¶

Table of Contents

Introduction¶

Goal¶

Install Libraries¶

pyspark¶

wordcloud¶

Setting Display configurations¶

Setting itable options¶

Importing Data¶

Spark¶

Why is important to use options?¶

Pandas¶

date type¶

dates format¶

Exploratory Data¶

Data info¶

Spark¶

Pandas¶

Netflix Data Null Values¶

Spark Nulls¶

Converting values to a tidy table: Why it matters?¶

Data nulls results¶

Bar Plot¶

Pie Plot¶

Words Cloud¶

Pandas Nulls¶

Data nulls results¶

Bar Plot¶

Pie Plot¶

Words Cloud¶

Nulls Data Merge¶

Plotly templates¶

Bar Plot¶

Line Plot¶

Comparing Pandas and Spark Data Processing: Key Findings and Insights¶

Line Plot¶

Words Cloud¶

Impute Null Values¶

Nulls Values Matrix¶

Exploring and Cleaning¶

No Null values¶

Nulls Values Matrix¶

Export to html¶